{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- encoding: utf-8 -*-\n",
    "import sys\n",
    "import codecs\n",
    "import csv\n",
    "import MeCab\n",
    "import os\n",
    "import re\n",
    "from collections import Counter,OrderedDict\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reload(sys)\n",
    "sys.setdefaultencoding('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 读取停用词一览\n",
    "def readStopWord(ssfile):\n",
    "    ss = []\n",
    "    fr = open(ssfile,\"r\")\n",
    "    for line in fr.readlines():\n",
    "        line = line.strip()\n",
    "        if line != '':\n",
    "            ss.append(line)\n",
    "    fr.close()\n",
    "    return ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 读取CSV文件\n",
    "def readDataFile(fileName):\n",
    "    with open(fileName,\"r+\") as csvfile:\n",
    "        reader = csv.reader(csvfile)\n",
    "        print \"Read CSV:\",fileName\n",
    "        # 读取内容\n",
    "        for line in reader:\n",
    "            return line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 获取词汇本体\n",
    "def get_surfaces(node):\n",
    "    words = []\n",
    "    nouns = []\n",
    "    while node:\n",
    "        word  = node.surface\n",
    "        words.append(word)\n",
    "        noun = node.feature.split(\",\")[0]\n",
    "        nouns.append(noun)\n",
    "        node = node.next\n",
    "    return words,nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 利用词性筛选词汇\n",
    "def select_feature(data,attr):\n",
    "    result = []\n",
    "    pattern1 = re.compile(r'[^0-9]+')\n",
    "    pattern2 = re.compile(ur'[^０１２３４５６７８９]+')\n",
    "    pattern3 = re.compile(ur'[^\\[\\]!$%&\\'\\\"\\(\\):\\-\\.,/;=<>]+')\n",
    "    pattern4 = re.compile(ur'[^！　％、。）※「」（＞～』＜？－．♪【⇒∞★〇・⇔]+')\n",
    "    # 需要根据词性做筛选\n",
    "    if attr is not None:\n",
    "        for (w,a) in data:\n",
    "            if a in attr:\n",
    "                tmp = w.decode('utf-8')\n",
    "                matcher1 = re.match(pattern1,w)\n",
    "                matcher2 = re.match(pattern2,tmp)\n",
    "                matcher3 = re.match(pattern3,tmp)\n",
    "                matcher4 = re.match(pattern4,tmp)\n",
    "                if (matcher1 is not None) and (matcher2 is not None) and (matcher3 is not None) and (matcher4 is not None):\n",
    "                    result.append((w,a))\n",
    "    else:\n",
    "        # 记号以外的词全部输出\n",
    "        for (w,a) in data:\n",
    "            if a != u'記号':\n",
    "                tmp = w.decode('utf-8')\n",
    "                matcher1 = re.match(pattern1,w)\n",
    "                matcher2 = re.match(pattern2,tmp)\n",
    "                matcher3 = re.match(pattern3,tmp)\n",
    "                matcher4 = re.match(pattern4,tmp)\n",
    "                if (matcher1 is not None) and (matcher2 is not None) and (matcher3 is not None) and (matcher4 is not None):\n",
    "                    result.append((w,a))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 把一个目录内的文件汇总以后写入一个文件\n",
    "def writeOutput(fileName,data,mode):\n",
    "    fw = open(fileName,mode)\n",
    "    line = \"\"\n",
    "    for w in data:\n",
    "        line = line + w + \"\\t\"\n",
    "    fw.writelines( line  + \"\\n\")\n",
    "    fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 把筛选以后的结果写入单个\n",
    "def writeSingleFile(fileName,data,mode):\n",
    "   \n",
    "    fw = open(fileName,mode)\n",
    "    line = \"\"\n",
    "    for item in data:\n",
    "        w,a = item\n",
    "        fw.writelines( w + \",\" + a + \"\\n\")\n",
    "    fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from collections import Counter,OrderedDict\n",
    "# Count结果，然后写入JSON文件\n",
    "# 抽取计数前 top_n\n",
    "def writeCountJSON(fileName,data,top_n = 10,sort=False,itemFilter=False,filterSet=None):    \n",
    "    count = Counter(data)\n",
    "    tmp = count.most_common(top_n)\n",
    "    # 排序\n",
    "    if sort :\n",
    "        print \" * dictionary sorted * \"\n",
    "        out = OrderedDict(sorted(tmp, key=lambda item:int(item[1]), reverse=False))\n",
    "    else:\n",
    "        out = dict(tmp)\n",
    "    # 过滤元素\n",
    "    if itemFilter:\n",
    "        if filterSet is not None:\n",
    "            for f in out.keys():\n",
    "                if f in filterSet:\n",
    "                    out.pop(f)\n",
    "    with open(fileName, 'w') as json_file:\n",
    "        json_file.write(json.dumps(out,sort_keys=False,indent=4, ensure_ascii=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 写入JSON文件\n",
    "def writeJSON(fileName,data):    \n",
    "    with open(fileName, 'w') as json_file:\n",
    "        json_file.write(json.dumps(data,indent=4, ensure_ascii=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 词性筛选 mode\n",
    "# 筛选 形容詞 ：mode 指定 \"A\"\n",
    "# 筛选 动词 ：mode 指定 \"V\"\n",
    "# 例如：筛选 名词，形容词的场合 :mode = \"NA\"\n",
    "def processFile(inPath,outPath,stopwords,mode):\n",
    "    \n",
    "    # 保存一个商品所有评论的分词\n",
    "    wordFreq = []\n",
    "    region = []\n",
    "    age = []\n",
    "    gender = []\n",
    "    car = []\n",
    "    fee = []\n",
    "    ### 情感倾向 \n",
    "    emotion = []\n",
    "    \n",
    "    # 根据输入的目录获得商品名\n",
    "    syouhin = inPath.split('/')[-2]\n",
    "    \n",
    "    # 词频统计结果\n",
    "    outFileFQ = outPath + \"freq_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 地域统计结果\n",
    "    outFileRegion = outPath + \"Region_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 年龄统计结果\n",
    "    outFileAge = outPath + \"Age_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 性别统计结果\n",
    "    outFileGender = outPath + \"Gender_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 车型统计结果\n",
    "    outFileCar = outPath + \"Car_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 保险费统计结果\n",
    "    outFileFee = outPath + \"Fee_\" + syouhin + \".json\"\n",
    "    \n",
    "    ### 情感统计结果\n",
    "    outFileEmotion = outPath + \"Emotion_\" + syouhin + \".json\"\n",
    "    \n",
    "    # 商品分词结果\n",
    "    outFile = outPath + \"Summary_\" + syouhin + \".txt\"\n",
    "    \n",
    "    # 全部商品分词结果合并\n",
    "    outFileAll = outPath + \"all_summary.txt\"\n",
    "    \n",
    "    # 全部评论合并\n",
    "    outFileMerge = outPath + \"all_merge.txt\"\n",
    "    \n",
    "    # 商品评论分词结果（mapreduce用）\n",
    "    outSinglePath = outPath + syouhin + \"_mapred/\"\n",
    "    if not os.path.exists(outSinglePath):\n",
    "        os.makedirs(outSinglePath) \n",
    "        \n",
    "    # 商品评论分词结果（mapreduce用）\n",
    "    outSingleJSON = outPath + syouhin + \"_json/\"\n",
    "    if not os.path.exists(outSingleJSON):\n",
    "        os.makedirs(outSingleJSON) \n",
    "    \n",
    "    # 获取文件列表\n",
    "    fileList = os.listdir(inPath)\n",
    "    for i in range(0,len(fileList)):\n",
    "        f = fileList[i]\n",
    "        jsonData = {}\n",
    "        path = os.path.join(inPath,f)\n",
    "        # 只对文件夹下的文件操作\n",
    "        if os.path.isfile(path):\n",
    "            print \"-----Processing File:\",f\n",
    "            # 编辑文件路径\n",
    "            fileName = inPath + f\n",
    "            # 获取类别(验证用)\n",
    "            catlog = f.split(\"_\")[0]\n",
    "            # 获取文件内容\n",
    "            content = readDataFile(fileName)\n",
    "            # 获取评论属性\n",
    "            info = catlog + \"#\" + content[1]\n",
    "            commentInfo = content[1].replace(\"（\",\"\").replace(\"）\",\"\").split(\"/\")\n",
    "            \n",
    "            # 编辑JSON文件内容\n",
    "            jsonData['product'] = catlog\n",
    "            jsonData['region'] = commentInfo[0]\n",
    "            jsonData['age'] = commentInfo[1]\n",
    "            jsonData['gender'] = commentInfo[2]\n",
    "            jsonData['car'] = commentInfo[3]\n",
    "            jsonData['fee'] = commentInfo[4]\n",
    "            jsonData['emotion'] = content[2]\n",
    "            jsonData['comment'] = content[0]\n",
    "            \n",
    "            # 加入list 用于统计\n",
    "            region.append(commentInfo[0]) # 地域\n",
    "            age.append(commentInfo[1])    # 年龄\n",
    "            gender.append(commentInfo[2]) # 性别\n",
    "            car.append(commentInfo[3]) # 车型\n",
    "            fee.append(commentInfo[4].split(\" \")[1])     # 保险费    \n",
    "            emotion.append(content[2]) # 情感\n",
    "            \n",
    "            # 定义分词器\n",
    "            mt = MeCab.Tagger('mecabrc')\n",
    "            node = mt.parseToNode(content[0])\n",
    "            words, nouns = get_surfaces(node)\n",
    "            wordList = zip(words,nouns)\n",
    "\n",
    "            attr = []\n",
    "            \n",
    "            if \"N\" in mode:\n",
    "                # 筛选名词结果写入CSV\n",
    "                attr.append(u'名詞')\n",
    "            \n",
    "            if \"A\" in mode:\n",
    "                # 筛选形容詞结果写入CSV\n",
    "                attr.append(u'形容詞')\n",
    "\n",
    "            if \"V\" in mode:\n",
    "                # 筛选動詞结果写入CSV\n",
    "                attr.append(u'動詞')\n",
    "            \n",
    "            if mode == 'all':\n",
    "                attr = None\n",
    "\n",
    "            # 筛选名词结果写入CSV\n",
    "            selectList = select_feature(wordList,attr)\n",
    "\n",
    "            # 分词后结果输出集合\n",
    "            outWord = []\n",
    "            # 评论者信息\n",
    "            outWord.append(info)\n",
    "            # 评论情感标志\n",
    "            outWord.append(content[2])\n",
    "            \n",
    "            # 分词前文本合并集合\n",
    "            outMerge = []\n",
    "            # 评论者情报\n",
    "            outMerge.append(info)\n",
    "            # 评论情感标志\n",
    "            outMerge.append(content[2])\n",
    "            # 评论本体\n",
    "            outMerge.append(content[0])\n",
    "            \n",
    "            mapred = []\n",
    "            \n",
    "            # 去掉停用词\n",
    "            for (w,a) in selectList:\n",
    "#                 print \"w=\",w\n",
    "                if w not in stopwords:\n",
    "                    outWord.append(w)\n",
    "                    mapred.append((w,a))\n",
    "                    wordFreq.append(w)\n",
    "            # 为每一个目录(商品)，汇总生成一个文件\n",
    "            writeOutput(outFile,outWord,\"a\")\n",
    "            \n",
    "            # 每一个CSV，生成一个分词文件，当作mapreduce的输入          \n",
    "            singleFile = outSinglePath + f.split('.')[0] + \"_mapred.txt\"\n",
    "            writeSingleFile(singleFile,mapred,\"wb\")\n",
    "            \n",
    "            # 每一个CSV，生成一个json文件\n",
    "            jsonFile = outSingleJSON + f.split('.')[0] + \"_info.json\"\n",
    "            writeJSON(jsonFile,jsonData)\n",
    "            \n",
    "            # 把所有的CSV文件的评论内容，合并到一个文件里\n",
    "            writeOutput(outFileMerge,outMerge,\"a\")\n",
    "            \n",
    "            # 把所有的CSV文件的分词筛选结果，写入一个文件里\n",
    "            writeOutput(outFileAll,outWord,\"a\")           \n",
    "\n",
    "        \n",
    "    # 计算一个商品所有评论的词频，写入文件\n",
    "    filterList = [u\"AIU\",u\"事故\",u\"保険料\",u\"対応\",u\"補償内容\",u\"車\",u\"保険\",u\"代理店\",u\"損保\",u\"補償\",\n",
    "                  u\"アクサ\",u\"契約\",u\"連絡\",u\"アクサダイレクト\",u\"イーデザイン\",u\"電話\",u\"保険会社\",\n",
    "                  u\"ニッセイ\",u\"ダイレクト\",u\"三井\",u\"サービス\",u\"三井住友海上\",u\"ジャパン\",u\"日新火災\",\n",
    "                  u\"加入\",u\"セゾン\",u\"チューリッヒ\",u\"担当者\",u\"担当\",u\"必要\",u\"会社\",u\"内容\",u\"車両保険\",\n",
    "                 u\"自動車保険\",u\"インターネット\",u\"ネット\"]\n",
    "    writeCountJSON(outFileFQ,wordFreq,top_n=40,sort=True,itemFilter=True,filterSet=filterList)\n",
    "      \n",
    "    writeCountJSON(outFileRegion,region,top_n=50)\n",
    "    writeCountJSON(outFileAge,age,top_n=10)\n",
    "    writeCountJSON(outFileGender,gender,top_n=10)\n",
    "    writeCountJSON(outFileCar,car,top_n=10)\n",
    "    writeCountJSON(outFileFee,fee,top_n=50)\n",
    "    writeCountJSON(outFileEmotion,emotion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def processAll():\n",
    "    # 读入停用词一览\n",
    "    ssfile = \"./conf/Japanese_stopwords.txt\"\n",
    "    ssList = readStopWord(ssfile)\n",
    "    \n",
    "    input_common = [\"./data/comment/aiu-sonpo/\",\n",
    "                      \"./data/comment/axa-direct/\",\n",
    "                      \"./data/comment/e-design/\",\n",
    "                      \"./data/comment/ioi-sonpo/\",\n",
    "                      \"./data/comment/mitsui-direct/\",\n",
    "                      \"./data/comment/ms-ins/\",\n",
    "                      \"./data/comment/nipponkoa/\",\n",
    "                      \"./data/comment/nisshin-kasai/\",\n",
    "                      \"./data/comment/saison/\",\n",
    "                      \"./data/comment/sonpo24/\",\n",
    "                      \"./data/comment/sony-sonpo/\",\n",
    "                      \"./data/comment/zurich/\"]\n",
    "    \n",
    "    input_lstm = [\"./data/comment_lstm/aiu-sonpo/\",\n",
    "                  \"./data/comment_lstm/axa-direct/\",\n",
    "                  \"./data/comment_lstm/e-design/\",\n",
    "                  \"./data/comment_lstm/ioi-sonpo/\",\n",
    "                  \"./data/comment_lstm/mitsui-direct/\",\n",
    "                  \"./data/comment_lstm/ms-ins/\",\n",
    "                  \"./data/comment_lstm/nipponkoa/\",\n",
    "                  \"./data/comment_lstm/nisshin-kasai/\",\n",
    "                  \"./data/comment_lstm/saison/\",\n",
    "                  \"./data/comment_lstm/sonpo24/\",\n",
    "                  \"./data/comment_lstm/sony-sonpo/\",\n",
    "                  \"./data/comment_lstm/zurich/\"]\n",
    "    \n",
    "    # lstm 用的输入文件\n",
    "    input_path = input_lstm \n",
    "    \n",
    "    \n",
    "    output_common = \"./data/processed/\"\n",
    "    output_lstm = \"./data/processed_lstm/\"\n",
    "    \n",
    "    # lstm\n",
    "    output_path = output_lstm\n",
    "    \n",
    "    if not os.path.exists(output_path):\n",
    "        os.makedirs(output_path)\n",
    "    \n",
    "    # 筛选名词    \n",
    "#     processFile(input_path,output_path,\"N\")\n",
    "    # 筛选名词和形容次\n",
    "#     processFile(input_path,output_path,\"NA\")\n",
    "    # 筛选名词和形容次，动词\n",
    "#     processFile(input_path,output_path,\"NAV\")\n",
    "    for p in input_path:\n",
    "        print \"Process Folder:\",p\n",
    "#         processFile(p,output_path,ssList,\"NA\")\n",
    "        processFile(p,output_path,ssList,\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def processTest():\n",
    "    \n",
    "    # 读入停用词一览\n",
    "    ssfile = \"./conf/Japanese_stopwords.txt\"\n",
    "    ssList = readStopWord(ssfile)\n",
    "       \n",
    "    input_path = [\"./data/comment/aiu-sonpo/\"] \n",
    "    \n",
    "    output_path = \"./data/test/\"\n",
    "    if not os.path.exists(output_path):\n",
    "        os.makedirs(output_path)\n",
    "    \n",
    "    # 筛选名词    \n",
    "#     processFile(input_path,output_path,\"N\")\n",
    "    # 筛选名词和形容次\n",
    "#     processFile(input_path,output_path,\"NA\")\n",
    "    # 筛选名词和形容次，动词\n",
    "#     processFile(input_path,output_path,\"NAV\")\n",
    "    for p in input_path:\n",
    "        print \"Process Folder:\",p\n",
    "        processFile(p,output_path,ssList,\"NA\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    \n",
    "    # 处理全体数据\n",
    "    processAll()\n",
    "    \n",
    "    # 处理测试数据\n",
    "#     processTest()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
